var amqp = require('amqplib/callback_api');
var rules = require('./rulestore/rules.json');
var _ = require('underscore');
var fs = require('fs');

amqp.connect('amqp://localhost', function(err, conn) {
  conn.createChannel(function(err, ch) {
    var tq = 'task_queue';
    var sq = 'save_queue';
    ch.assertQueue(tq, {durable: true});
    ch.assertQueue(sq, {durable: true});
    ch.prefetch(1);
    console.log(" [*] Waiting for tasks in %s. To exit press CTRL+C", sq);
    ch.consume(sq, function(msg) {
      var task = JSON.parse(msg.content.toString());
      console.log(" [x] Received %s,%s",task.url,task.path);
      parse(task,function(e,links) {
        if(e){
          console.log('parse fail,resend to '+sq);
          ch.sendToQueue(sq,msg,{persistent:true});
        }else{
          links.forEach(function(l) {
            var request ={};
            request.isDynamic = false;
            request.url = l;
            console.log('send to '+tq+':'+l);
            ch.sendToQueue(tq,new Buffer(JSON.stringify(request)),{persistent:true});
          })
          console.log('**********finish parse');
          ch.ack(msg);
        }
      })
    }, {noAck: false});
  });
});

function parse(task,cb) {
  console.log('start to parse');
  var url = task.url;
  var filepath = task.path;
  var rule = _.find(rules,function(rule) {
    var patterns = rule.url_pattern;
    for(var i =0;i<patterns.length;i++){
      var reg = new RegExp(patterns[i]);
      var flag = reg.test(url);
      if(flag){
        return true;
      }
    }
    return false;
  });
  if(!rule){
    return cb(null,[]);
  }
  var link_rule = rule.link_parse_rule;
  if(!link_rule){
    return cb(null,[]);
  }
  var links = [];
  fs.readFile(filepath,function(e,html) {
    if(e)
      return cb(e);
    var linkReg = new RegExp(link_rule,'g');
    var res = null;
    while (res = linkReg.exec(html)) {
      links.push(res[1]);
    }
    return cb(null,links);
    // fs.writeFile(__dirname+'/links/link.json',JSON.stringify(links),function(e) {
    //   return cb(null,links);
    // })
  })

}

// var Xray = require('x-ray');
// var x = Xray();
//
// x('http://s.wanfangdata.com.cn/Paper.aspx?q=docker&f=top','.record-item',[{
//   title:'.record-title .title',
//   href:'.record-title .title@href',
//   abstract:'.record-desc',
//   keyword:'.record-keyword',
// }]).paginate('p.pager a.page:eq(-1)@href')//怎么选最后一个？
// .limit('10')
// .write('./xrayResult/results.json');
